#!/usr/bin/env ruby

# Copyright (c) 2021-2025 Huawei Device Co., Ltd.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

macro(:load_diff) do |ptr1, ptr2, offset|
    Xor(LoadI(ptr1).Imm(Constants::STRING_DATA_OFFSET + " + " + offset.to_s).u64,
        LoadI(ptr2).Imm(Constants::STRING_DATA_OFFSET + " + " + offset.to_s).u64).u64
end

scoped_macro(:unpack_length) do |length, compression, length_shift|
    if compression
        char_length := ShrI(length).Imm(length_shift).u64
        not_compressed := AndI(length).Imm(1).u64
        unpacked_length := Shl(char_length, not_compressed).u64
    elsif length_shift != 0
        unpacked_length := ShrI(length).Imm(length_shift).u64
    end
end

def GenerateStringEquals(lang, dynamic, compression)
    suffix = (compression ? "Compressed" : "")
    length_shift = (dynamic ? 2 : 1)
    mode = [:FastPath]
    mode.push(:DynamicMethod, :DynamicStub) if dynamic

    reg_mask = Options.arch == :arm64 ?
        RegMask.new($full_regmap, :arg0, :arg1, :callee0, :callee1, :callee2, :callee3, :callee4, :callee5, :callee6, :callee7, :tmp0, :tmp1) :
        $panda_mask + :tmp0 + :tmp1 + :tr

    function("#{lang}StringEqualsUnroll#{suffix}".to_sym,
          params: {str1_ref: 'ref', str2_ref: 'ref'},
          regmap: $full_regmap,
          regalloc_set: reg_mask,
          mode: mode,
          lang: lang.empty? ? 'PANDA_ASSEMBLY' : lang.upcase) {
        if Options.arch == :arm32
            Intrinsic(:UNREACHABLE).void.Terminator
            next
        end
        str1 := Cast(str1_ref).SrcType(Constants::COMPILER_REFERENCE).ptr
        str2 := Cast(str2_ref).SrcType(Constants::COMPILER_REFERENCE).ptr
        # Compare first 32 bytes
        buf1 := LoadI(str1).Imm(Constants::STRING_DATA_OFFSET).u64
        buf2 := LoadI(str2).Imm(Constants::STRING_DATA_OFFSET).u64
        If(buf1, buf2).NE.Unlikely.b {
            Goto(:NotEqual)
        }
        diff := load_diff(str1, str2, 8)
        diff := Or(diff, load_diff(str1, str2, 16)).u64
        diff := Or(diff, load_diff(str1, str2, 24)).u64
        If(diff, 0).NE.Unlikely.b {
            Goto(:NotEqual)
        }
        length := LoadI(str1).Imm(Constants::STRING_LENGTH_OFFSET).u32
        length := Cast(length).SrcType("DataType::UINT32").u64
        length := unpack_length(length, compression, length_shift)
        last_buf_index := SubI(length).Imm(32).u64
        last_ptr1 := Add(str1, last_buf_index).ptr
        last_ptr2 := Add(str2, last_buf_index).ptr
        first_ptr1 := AddI(str1).Imm(32).ptr
        first_ptr2 := AddI(str2).Imm(32).ptr
    Label(:Loop)
        ptr1 := Phi(first_ptr1, next_ptr1).ptr
        ptr2 := Phi(first_ptr2, next_ptr2).ptr
        diff := load_diff(ptr1, ptr2, 0)
        diff := Or(diff, load_diff(ptr1, ptr2, 8)).u64
        diff := Or(diff, load_diff(ptr1, ptr2, 16)).u64
        diff := Or(diff, load_diff(ptr1, ptr2, 24)).u64
        If(diff, 0).NE.Unlikely.b {
            Goto(:NotEqual)
        }
        next_ptr1 := Add(ptr1, 32).ptr
        If(next_ptr1, last_ptr1).GE.Unlikely.b {
            diff := load_diff(last_ptr1, last_ptr2, 0)
            diff := Or(diff, load_diff(last_ptr1, last_ptr2, 8)).u64
            diff := Or(diff, load_diff(last_ptr1, last_ptr2, 16)).u64
            diff := Or(diff, load_diff(last_ptr1, last_ptr2, 24)).u64
            If(diff, 0).NE.Unlikely.b {
                Goto(:NotEqual)
            }
            Return(1).b
        }
        next_ptr2 := Add(ptr2, 32).ptr
        Goto(:Loop)
    Label(:NotEqual)
        Return(0).b
    }

    reg_mask = Options.arch == :arm64 ? RegMask.new($full_regmap, :arg0, :arg1, :tmp0, :tmp1, :callee0, :callee2) :
        RegMask.new($full_regmap, :arg0, :arg1, :tmp0, :tmp1, :callee0, :caller1)

    function("#{lang}StringEquals#{suffix}".to_sym,
            params: {str1: 'ref', str2: 'ref'},
            regmap: $full_regmap,
            regalloc_set: reg_mask,
            mode: mode,
            lang: lang.empty? ? 'PANDA_ASSEMBLY' : lang.upcase) {
        # Arm32 is not supported
        # length shift should be from 0 to 2
        if Options.arch == :arm32 || length_shift > 2
            Intrinsic(:UNREACHABLE).void.Terminator
            next
        end
        unless dynamic
            If(str2, 0).EQ.Unlikely.b {
                Goto(:NotEqual)
            }
        end
        If(str1, str2).EQ.Unlikely.b {
            Return(1).b
        }
        if dynamic
            length1 := LoadI(str1).Imm(Constants::STRING_LENGTH_OFFSET).u32
            length2 := LoadI(str2).Imm(Constants::STRING_LENGTH_OFFSET).u32
            length1 := AndI(length1).Imm("~(2U)").u32
            length2 := AndI(length2).Imm("~(2U)").u32
            If(length1, length2).NE.Unlikely.b {
                Goto(:NotEqual)
            }
            length := Cast(length1.u32).u64
        else
            class1 := load_class(str1)
            class2 := load_class(str2)
            If(class1, class2).NE.Unlikely.b {
                Goto(:NotEqual)
            }
            length1 := LoadI(str1).Imm(Constants::STRING_LENGTH_OFFSET).u32
            length2 := LoadI(str2).Imm(Constants::STRING_LENGTH_OFFSET).u32
            If(length1, length2).NE.Unlikely.b {
                Goto(:NotEqual)
            }
            length := Cast(length1.u32).u64
        end
        length := unpack_length(length, compression, length_shift)
        If(length, 8).GT.b {
            Goto(:Long)
        }
        odd_bytes := Sub(8, length).u64
        last_idx := Sub(Constants::STRING_DATA_OFFSET, odd_bytes).u64
        buf1 := Load(str1, last_idx).u64
        buf2 := Load(str2, last_idx).u64
        odd_bits := Shl(odd_bytes, 3).u64
        diff := Shr(Xor(buf1, buf2).u64, odd_bits).u64
        res := Compare(diff, 0).EQ.b
        if dynamic
            # If length is 0, odd_bits is 64 and Shr above does nothing, so we effectively compare last 8 bytes of two strings
            # before their data (length and hash code). Hash code for empty string is always 0, but value stored in length field
            # of equal strings can be different in dynamic implementation, so we check (length == 0) separately
            res := Or(res, Compare(length, 0).EQ.b).b
        end
        Return(res).b

    Label(:Long)
        unroll := Compare(length, 64).GE.b
        IfImm(unroll).Imm(0).SrcType("DataType::BOOL").NE.b {
            LiveOut(str1).DstReg(regmap[:arg0]).ref
            LiveOut(str2).DstReg(regmap[:arg1]).ref
            entrypoint_id = "#{lang.empty? ? '' : lang.upcase + '_'}STRING_EQUALS_UNROLL" + (compression ? "_COMPRESSED" : "");
            entrypoint_offset = get_entrypoint_offset(entrypoint_id)
            entrypoint_name = "#{lang.empty? ? '' : lang}StringEqualsUnroll" + (compression ? "Compressed" : "");
            Intrinsic(:TAIL_CALL).AddImm(entrypoint_offset).MethodAsImm(entrypoint_name).Terminator.b
        }
        first_idx := Constants::STRING_DATA_OFFSET
        last_idx := Sub(Add(first_idx, length).u64, 8).u64

    Label(:Loop)
        idx := Phi(first_idx, next_idx).u64
        buf1 := Load(str1, idx).u64
        buf2 := Load(str2, idx).u64
        If(buf1, buf2).NE.Unlikely.b {
            Goto(:NotEqual)
        }
        next_idx := Add(idx, 8).u64
        If(next_idx, last_idx).GE.Unlikely.b {
            buf1 := Load(str1, last_idx).u64
            buf2 := Load(str2, last_idx).u64
            res := Compare(buf1, buf2).EQ.b
            Return(res).b
        }
        Goto(:Loop)
    Label(:NotEqual)
        Return(0).b
    }
end

# Try to allocate String in TLAB.
# The result is either a pointer to a new string or null if there is no enough space in TLAB.
macro(:allocate_string_tlab) do |string_klass, data_size|
  if Options.arch == :arm32
    Intrinsic(:UNREACHABLE).Terminator.void
    ReturnVoid().void
    next
  end

  # Add sizeof(String) and do align
  _data_size := Cast(data_size).word
  _size := AndI(AddI(_data_size).Imm(Constants::STRING_CLASS_SIZE_WITH_ALIGNMENT).word).Imm(Constants::ALIGNMENT_MASK).word
  # Load pointer to the TLAB from TLS
  _tlab := LoadI(%tr).Imm(Constants::TLAB_OFFSET).ptr
  # Load pointer to the start address of free memory in the TLAB
  _start := LoadI(_tlab).Imm(Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr
  # Load pointer to the end address of free memory in the TLAB
  _end := LoadI(_tlab).Imm(Constants::TLAB_MEMORY_END_ADDR_OFFSET).ptr
  # Check if there is enough space
  If(Sub(_end, _start).word, _size).B.Unlikely.b {
    Goto(:SlowPathEntrypoint)
  }
  Intrinsic(:WRITE_TLAB_STATS_SAFE, _start, _size, Cast(-1).u64).void if defines.DEBUG
  if defines.__SANITIZE_ADDRESS__ || defines.__SANITIZE_THREAD__
    call_runtime_save_all(Constants::ANNOTATE_SANITIZERS_NO_BRIDGE, _start, _size).void
  end
  # Store class of the object
  store_class(_start, string_klass)
  # Update the TLAB state
  StoreI(Add(_tlab, Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr, Add(_start, _size).ptr).Imm(0).Volatile.ptr
  # Return a pointer to the newly allocated string
  _allocated_string := _start
end

# Try to allocate String in TLAB.
# The result is either a pointer to a new string or null if there is no enough space in TLAB.
# this is a version that does not do allocation tracking due to the bug #24977
macro(:allocate_string_tlab_no_debug) do |string_klass, data_size|
  if Options.arch == :arm32
    Intrinsic(:UNREACHABLE).Terminator.void
    ReturnVoid().void
    next
  end

  # Add sizeof(String) and do align
  _data_size := Cast(data_size).word
  _size := AndI(AddI(_data_size).Imm(Constants::STRING_CLASS_SIZE_WITH_ALIGNMENT).word).Imm(Constants::ALIGNMENT_MASK).word
  # Load pointer to the TLAB from TLS
  _tlab := LoadI(%tr).Imm(Constants::TLAB_OFFSET).ptr
  # Load pointer to the start address of free memory in the TLAB
  _start := LoadI(_tlab).Imm(Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr
  # Load pointer to the end address of free memory in the TLAB
  _end := LoadI(_tlab).Imm(Constants::TLAB_MEMORY_END_ADDR_OFFSET).ptr
  # Check if there is enough space
  If(Sub(_end, _start).word, _size).B.Unlikely.b {
    Goto(:SlowPathEntrypoint)
  }
  if defines.__SANITIZE_ADDRESS__ || defines.__SANITIZE_THREAD__
    call_runtime_save_all(Constants::ANNOTATE_SANITIZERS_NO_BRIDGE, _start, _size).void
  end
  # Store class of the object
  StoreI(_start, string_klass).Imm(Constants::OBJECT_CLASS_OFFSET).ref
  # Update the TLAB state
  StoreI(Add(_tlab, Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr, Add(_start, _size).ptr).Imm(0).Volatile.ptr
  # Return a pointer to the newly allocated string
  _allocated_string := _start
end

# Try to allocate Array of i8 chars in TLAB.
# The result is either a pointer to a new array or null if there is no enough space in TLAB.
macro(:allocate_array_of_bytes_tlab) do |array_klass, char_count|
  if Options.arch == :arm32
    Intrinsic(:UNREACHABLE).Terminator.void
    ReturnVoid().void
    next
  end

  elements_num := And(char_count, "0x00000000ffffffff").word
  # Account for u16 char size
  _size := Cast(elements_num).word
  # Add sizeof(Array) and do align
  _size := AndI(AddI(_size).Imm(Constants::ARRAY_CLASS_SIZE_WITH_ALIGNMENT).word).Imm(Constants::ALIGNMENT_MASK).word
  # Load pointer to the TLAB from TLS
  _tlab := LoadI(%tr).Imm(Constants::TLAB_OFFSET).ptr
  # Load pointer to the start address of free memory in the TLAB
  _start := LoadI(_tlab).Imm(Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr
  # Load pointer to the end address of free memory in the TLAB
  _end := LoadI(_tlab).Imm(Constants::TLAB_MEMORY_END_ADDR_OFFSET).ptr
  # Check if there is enough space
  If(Sub(_end, _start).word, _size).B.Unlikely.b {
    Goto(:SlowPathEntrypoint)
  }
  Intrinsic(:WRITE_TLAB_STATS_SAFE, _start, _size, Cast(-1).u64).void if defines.DEBUG
  if defines.__SANITIZE_ADDRESS__ || defines.__SANITIZE_THREAD__
    call_runtime_save_all(Constants::ANNOTATE_SANITIZERS_NO_BRIDGE, _start, _size).void
  end
  # Store class of the object
  store_class(_start, array_klass)
  # Store array length
  StoreI(_start, elements_num).Imm(Constants::ARRAY_LENGTH_OFFSET).word
  # Update the TLAB state
  StoreI(Add(_tlab, Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr, Add(_start, _size).ptr).Imm(0).Volatile.ptr
  # Return a pointer to the newly allocated array
  _allocated_array := _start
end

# Try to allocate Array of u16 chars in TLAB.
# The result is either a pointer to a new array or null if there is no enough space in TLAB.
macro(:allocate_array_of_chars_tlab) do |array_klass, char_count|
  if Options.arch == :arm32
    Intrinsic(:UNREACHABLE).Terminator.void
    ReturnVoid().void
    next
  end

  elements_num := And(char_count, "0x00000000ffffffff").word
  # Account for u16 char size
  _size := Shl(elements_num, 1).word
  # Add sizeof(Array) and do align
  _size := AndI(AddI(_size).Imm(Constants::ARRAY_CLASS_SIZE_WITH_ALIGNMENT).word).Imm(Constants::ALIGNMENT_MASK).word
  # Load pointer to the TLAB from TLS
  _tlab := LoadI(%tr).Imm(Constants::TLAB_OFFSET).ptr
  # Load pointer to the start address of free memory in the TLAB
  _start := LoadI(_tlab).Imm(Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr
  # Load pointer to the end address of free memory in the TLAB
  _end := LoadI(_tlab).Imm(Constants::TLAB_MEMORY_END_ADDR_OFFSET).ptr
  # Check if there is enough space
  If(Sub(_end, _start).word, _size).B.Unlikely.b {
    Goto(:SlowPathEntrypoint)
  }
  Intrinsic(:WRITE_TLAB_STATS_SAFE, _start, _size, Cast(-1).u64).void if defines.DEBUG
  if defines.__SANITIZE_ADDRESS__ || defines.__SANITIZE_THREAD__
    call_runtime_save_all(Constants::ANNOTATE_SANITIZERS_NO_BRIDGE, _start, _size).void
  end
  # Store class of the object
  store_class(_start, array_klass)
  # Store array length
  StoreI(_start, elements_num).Imm(Constants::ARRAY_LENGTH_OFFSET).word
  # Update the TLAB state
  StoreI(Add(_tlab, Constants::TLAB_CUR_FREE_POSITION_OFFSET).ptr, Add(_start, _size).ptr).Imm(0).Volatile.ptr
  # Return a pointer to the newly allocated array
  _allocated_array := _start
end


def GenerateCreateStringFromStringTlab(string_compression_enabled)
  suffix = (string_compression_enabled ? "Compressed" : "")
  available_regs = $panda_mask
  function("CreateStringFromStringTlab#{suffix}".to_sym,
            params: {str: 'ref'},
            regmap: $full_regmap,
            regalloc_set: available_regs,
            mode: [:FastPath]) {

    if Options.arch == :arm32
      Intrinsic(:UNREACHABLE).Terminator.void
      ReturnVoid().void
      next
    end

    # There is no check of the argument against NullPointer as
    # it's done in the InstBuilder (see AddArgNullcheckIfNeeded)
    klass := load_class(str)
    length := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32
    hashcode := LoadI(str).Imm(Constants::STRING_HASHCODE_OFFSET).u32
    data_size := unpack_length(Cast(length).u64, string_compression_enabled, 1)

    new_str := allocate_string_tlab(klass, Cast(data_size).word)
    StoreI(new_str, length).Imm(Constants::STRING_LENGTH_OFFSET).u32
    StoreI(new_str, hashcode).Imm(Constants::STRING_HASHCODE_OFFSET).u32

    # Copy string data
    src_str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).u64).ptr
    dst_str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr
    offs := Cast(0).u64
    If(data_size, 8).AE.Likely.b {
      stop := AndI(data_size).Imm(7).u64
    Label(:CopyLoop_8b)
      offs1 := Phi(offs, offs2).u64
      Store(dst_str_data, offs, Load(src_str_data, offs).u64).u64
      offs2 := AddI(offs1).Imm(8).u64
      If(offs2, stop).B.Likely.b {
        Goto(:CopyLoop_8b)
      }
    }
    offs3 := Phi(offs, offs2).u64
    If(offs3, data_size).B.Likely.b {
    Label(:CopyLoop_1b)
      offs4 := Phi(offs3, offs5).u64
      Store(dst_str_data, offs4, Load(src_str_data, offs4).u8).u8
      offs5 := AddI(offs4).Imm(1).u64
      If(offs5, data_size).B.Likely.b {
        Goto(:CopyLoop_1b)
      }
    }

    # String is supposed to be a constant object, so all its data should be visible by all threads
    Intrinsic(:DATA_MEMORY_BARRIER_FULL).void
    Return(new_str).ptr

  Label(:SlowPathEntrypoint)
    entrypoint = get_entrypoint_offset("CREATE_STRING_FROM_STRING_SLOW_PATH")
    Intrinsic(:SLOW_PATH_ENTRY, str).AddImm(entrypoint).MethodAsImm("CreateStringFromStringOddSavedBridge").Terminator.ptr
    Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG
  }
end  # def GenerateCreateStringFromStringTlab


###
# Checks if starting from arr_data the specified number of chars (char_count) can be compressed
#
# Utf16 char is ASCII if (utf16_char - 1U < utf::UTF8_1B_MAX)
# See runtime/include/coretypes/string.h - IsASCIICharacter
#
scoped_macro(:is_array_of_compressable_chars) do |arr_data, char_count|
  # Check 4-chars block at once if it's possible
  n_blocks := ShrI(char_count).Imm(2).u64
  If(n_blocks, 0).A.Likely.b {
    # 0x007F is utf::UTF8_1B_MAX
    utf8_1b_max := Cast(0x007F007F007F007F).u64
    utf8_1b_max_mask := Not(utf8_1b_max).u64
    i1 := Cast(0).u64
  Label(:CanBeCompressedLoop_4chars)
    i2 := Phi(i1, i3).u64
    four_chars := Load(arr_data, ShlI(i2).Imm(3).u64).u64
    # First: check if there are chars greater than utf::UTF8_1B_MAX
    If(And(four_chars, utf8_1b_max_mask).u64, 0).NE.Unlikely.b {
        compressable1 := Cast(0).b
        Goto(:CanBeCompressedLoopDone)
    }
    # Second: check if there are chars equal to zero
    If(And(SubI(four_chars).Imm(0x0001000100010001).u64, utf8_1b_max_mask).u64, 0).NE.Unlikely.b {
        compressable2 := Cast(0).b
        Goto(:CanBeCompressedLoopDone)
    }
    i3 := AddI(i2).Imm(1).u64
    If(i3, n_blocks).B.Likely.b {
        Goto(:CanBeCompressedLoop_4chars)
    }
  }
  # check the rest of the chars
  If(AndI(char_count).Imm(3).u64, 0).A.Likely.b {
    i4 := ShlI(n_blocks).Imm(2).u64  # number of already copied chars if any
Label(:CanBeCompressedLoop)
    i5 := Phi(i4, i6).u64
    ch := Load(arr_data, ShlI(i5).Imm(1).u64).u16
    If(SubI(ch).Imm(Constants::STRING_MUTF8_1B_MIN).u16, Cast(Constants::STRING_MUTF8_1B_MAX).u16).AE.Unlikely.b {
      compressable3 := Cast(0).b
      Goto(:CanBeCompressedLoopDone)
    }
    i6 := AddI(i5).Imm(1).u64
    If(i6, char_count).B.Likely.b {
      Goto(:CanBeCompressedLoop)
    }
  }
  compressable4 := Cast(1).b
Label(:CanBeCompressedLoopDone)
  compressable := Phi(compressable1, compressable2, compressable3, compressable4).b
end  # is_array_of_compressable_chars


###
# Copy u16 chars compressing them to u8.
# It is assumed that all u16 chars are compressable.
#
scoped_macro(:compress_u16_to_u8_chars) do |src_data, dst_data, char_count|
if Options.arch == :arm64
  # Copy 32-byte chunks (if any) compressing them into 16-byte chunks
  If(char_count, 16).AE.Likely.b {
    stop := Add(src_data, ShlI(ShrI(char_count).Imm(4).u64).Imm(5).u64).ptr
Label(:CopyLoop_32b)
    src_data1 := Phi(src_data, src_data2).ptr
    dst_data1 := Phi(dst_data, dst_data2).ptr
    Intrinsic(:COMPRESS_SIXTEEN_UTF16_TO_UTF8_CHARS_USING_SIMD, src_data1, dst_data1).void
    src_data2 := AddI(src_data1).Imm(32).ptr
    dst_data2 := AddI(dst_data1).Imm(16).ptr
    If(src_data2, stop).LT.Likely.b {
      Goto(:CopyLoop_32b)
    }
    char_count1 := AndI(char_count).Imm(0xF).u64
  }
  src_data3 := Phi(src_data, src_data2).ptr
  dst_data3 := Phi(dst_data, dst_data2).ptr
  char_count2 := Phi(char_count, char_count1).u64

  # Copy 16-byte chunk (if any) compressing it into 8-byte chunk
  If(char_count2, 8).AE.Likely.b {
    Intrinsic(:COMPRESS_EIGHT_UTF16_TO_UTF8_CHARS_USING_SIMD, src_data3, dst_data3).void
    src_data4 := AddI(src_data3).Imm(16).ptr
    dst_data4 := AddI(dst_data3).Imm(8).ptr
  }
  src_data5 := Phi(src_data3, src_data4).ptr
  dst_data5 := Phi(dst_data3, dst_data4).ptr
  # Copy 2-byte chunks compressing them into 1-byte
  n_2b := AndI(char_count2).Imm(7).u64
  If(n_2b, 0).A.Likely.b {
    j1 := Cast(0).u64
Label(:CopyLoop_2b)
    j := Phi(j1, j2).u64
    Store(dst_data5, j, Load(src_data5, ShlI(j).Imm(1).u64).u8).u8
    j2 := AddI(j).Imm(1).u64
    If(j2, n_2b).B.Likely.b {
      Goto(:CopyLoop_2b)
    }
  }
else  # if Options.arch == :arm64
  # Copy 8-byte chunks compressing them into 4-byte chunks
  n_8b := ShrI(char_count).Imm(2).u64
  If(n_8b, 0).A.Likely.b {
    i1 := Cast(0).u64
Label(:CopyLoop_8b)
    i := Phi(i1, i2).u64
    chunk := Load(src_data, ShlI(i).Imm(3).u64).u64
    chunk := AndI(chunk).Imm(0x00ff00ff00ff00ff).u64 # zero high part of each two-byte
    chunk := AndI(Or(chunk, ShrI(chunk).Imm(8).u64).u64).Imm(0x0000ffff0000ffff).u64
    chunk := Or(chunk, ShrI(chunk).Imm(16).u64).u64
    Store(dst_data, ShlI(i).Imm(2).u64, Cast(chunk).u32).u32
    i2 := AddI(i).Imm(1).u64
    If(i2, n_8b).B.Likely.b {
      Goto(:CopyLoop_8b)
    }
  }
  # Copy 2-byte chunks compressing them into 1-byte
  If(AndI(char_count).Imm(3).u64, 0).A.Likely.b {
    j1 := ShlI(n_8b).Imm(2).u64  # number of already copied chars if any
Label(:CopyLoop_2b)
    j := Phi(j1, j2).u64
    Store(dst_data, j, Load(src_data, ShlI(j).Imm(1).u64).u8).u8
    j2 := AddI(j).Imm(1).u64
    If(j2, char_count).B.Likely.b {
      Goto(:CopyLoop_2b)
    }
  }
end  # if Options.arch == :arm64
end  # compress_u16_to_u8_chars

###
# Copy dwords (8-bytes)
#
scoped_macro(:copy_dwords) do |src, dst, size|
    count := AndI(size).Imm(~0x7).word
    i1 := Cast(0).word
Label(:CopyLoop_8b)
    i := Phi(i1, i2).word
    If(i, count).AE.Unlikely {
        Goto(:End)
    }
    Store(dst, i, Load(src, i).word).word
    i2 := AddI(i).Imm(8).word
    Goto(:CopyLoop_8b)
Label(:End)
end # copy_dwords

###
# Copy u8 chars from src to dst
#
scoped_macro(:copy_u8_chars) do |src, dst, count|
    # Copy 8-byte chunks
    len := Cast(count).word
    copy_dwords(src, dst, len)

    # copy the tail if needed
    i1 := AndI(len).Imm(~0x7).word
Label(:CopyLoop)
    i := Phi(i1, i2).word
    If(i, len).AE.Unlikely {
        Goto(:End)
    }
    Store(dst, i, Load(src, i).u8).u8
    i2 := AddI(i).Imm(1).word
    Goto(:CopyLoop)

Label(:End)
end  # copy_u8_chars

###
# Copy u16 chars from src to dst
#
scoped_macro(:copy_u16_chars) do |src, dst, count|
    # Copy 8-byte chunks
    len := Cast(ShlI(count).Imm(1).u32).word
    copy_dwords(src, dst, len)

    # copy the tail if needed
    i1 := AndI(len).Imm(-8).u64
Label(:CopyLoop)
    i := Phi(i1, i2).u64
    If(i, len).AE.Unlikely.b {
        Goto(:End)
    }
    Store(dst, i, Load(src, i).u16).u16
    i2 := AddI(i).Imm(2).u64
    Goto(:CopyLoop)

Label(:End)
end  # copy_u16_chars

###
# Copy u8 chars expanding them to u16 chars
#
scoped_macro(:expand_u8_to_u16_chars) do |src, dst, count|
  i0 := Cast(0).u64
  len := Cast(count).u64
Label(:CopyLoop)
  i := Phi(i0, i1).u64
  If(i, len).AE.Unlikely.b {
      Goto(:End)
  }
  Store(dst, ShlI(i).Imm(1).u64, Cast(Load(src, i).u8).u16).u16
  i1 := AddI(i).Imm(1).u64
  Goto(:CopyLoop)
Label(:End)
end  # expand_u8_to_u16_chars

###
# Copy data from compressed string to array of utf16 chars
#
scoped_macro(:copy_compressed_string_to_array_of_chars) do |str_data, arr_data, char_count|
  # String contains 8-bit chars
  If(char_count, 0).A.Likely.b {
    i1 := Cast(0).u64
Label(:CopyLoop)
    i := Phi(i1, i2).u64
    Store(arr_data, ShlI(i).Imm(1).u64, Cast(Load(str_data, i).u8).u16).u16
    i2 := AddI(i).Imm(1).u64
    If(i2, char_count).B.Likely.b {
      Goto(:CopyLoop)
    }
  }
end  # copy_compressed_string_to_array_of_chars

# calculate the difference between string chunks
# stored in 64-bits numbers, byte/hword swap is
# required before comparison for the correct result

scoped_macro(:calculate_chars_difference_lat) do |d1, d2|
  ldata1 := Intrinsic(:REVERSE_BYTES_U64, d1).u64
  ldata2 := Intrinsic(:REVERSE_BYTES_U64, d2).u64
  ret := Cmp(ldata1, ldata2).i32
end

scoped_macro(:calculate_chars_difference_utf) do |d1, d2|
  udata1 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, Bitcast(d1).f64).f64).u64
  udata2 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, Bitcast(d2).f64).f64).u64
  ret := Cmp(udata1, udata2).i32
end

scoped_macro(:calculate_chars_difference) do |d1, d2, utf|
  data1 := Cast(d1).u64
  data2 := Cast(d2).u64

  If(utf, 0).EQ.b {
    cmp1 := calculate_chars_difference_lat(data1, data2)
  } Else {
    cmp2 := calculate_chars_difference_utf(data1, data2)
  }
  ret := Phi(cmp1, cmp2).i32
end

scoped_macro(:compare_LU_strings) do |lat, utf, len|
  # considering the data buffer to be allocated to
  # 8-bytes alignment, we can safely read 8-bytes chunks
  last_idx := SubI(len).Imm(4).i64
  i1 := 0
Label(:Loop)
  i := Phi(i1, i2).i64
  If(i, last_idx).GE.Unlikely.b {
    # can safely read 8 bytes behind - length and hashcode
    junk_bits := ShlI(Sub(i, last_idx).u64).Imm(4).u64
    last_lv := Load(lat, last_idx).u32
    last_v := Shr(Bitcast(Intrinsic(:EXPAND_U8_TO_U16, Bitcast(last_lv).f32).f64).u64, junk_bits).u64
    last_u := Shr(Load(utf, ShlI(last_idx).Imm(1).u64).u64, junk_bits).u64
    If(last_v, last_u).NE.b {
      Goto(:NotEqual)
    }
    Goto(:End)
  }

  lv := Load(lat, i).u32
  v := Bitcast(Intrinsic(:EXPAND_U8_TO_U16, Bitcast(lv).f32).f64).u64
  u := Load(utf, ShlI(i).Imm(1).u64).u64
  If(v, u).NE.b {
    Goto(:NotEqual)
  }
  i2 := AddI(i).Imm(4).i64
  Goto(:Loop)

  # the version assuming 128-bits types support
  #
  # v := Intrinsic(:EXPAND_U8_TO_U16, Load(lat, i).f64).f128
  # u := Load(utf, i).f128
  # v1 = Intrinsic(:GET_LOW_PART, v).u64
  # u1 = Intrinsic(:GET_LOW_PART, u).u64 
  # n11 := Bitcast(v1).u64
  # n21 := Bitcast(u1).u64
  # If(n11, n21).NE.b {
  #   d11 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, v1).f64).u64
  #   d21 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, u1).f64).u64
  #   Goto(:NotEqual)
  # }
  # v2 = Intrinsic(:GET_HIGH_PART, v).u64
  # u2 = Intrinsic(:GET_HIGH_PART, u).u64 
  # n12 := Bitcast(v2).u64
  # n22 := Bitcast(u2).u64
  # If(n12, n22).NE.b {
  #   d12 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, v2).f64).u64
  #   d22 := Bitcast(Intrinsic(:REVERSE_HALF_WORDS, u2).f64).u64
  #   Goto(:NotEqual)
  # }
  #
  # Label(:NotEqual)
  # d1 := Phi(d11, d12)
  # d2 := Phi(d21, d22)
  # res := Sub(d1, d2).u64

Label(:NotEqual)
  v_ne := Phi(last_v, v).u64
  u_ne := Phi(last_u, u).u64
  ret_ne := calculate_chars_difference_utf(v_ne, u_ne).i32
  Goto(:End)

Label(:End)
  # -1: the prefix is the same but the utf string has to be
  # longer as it would have had latin encoding otherwise
  ret := Phi(-1, ret_ne).i32
end

scoped_macro(:compare_mixed_strings) do |buf1, buf2, len, utf_is_first|
  If(utf_is_first, 0).EQ.b {
    ret1 := compare_LU_strings(buf1, buf2, len)
  } Else {
    ret2 := Neg(compare_LU_strings(buf2, buf1, len).i32).i32
  }
  ret := Phi(ret1, ret2).i32
end

def GenerateCreateStringFromCharArrayTlab(string_compression_enabled)
  suffix = (string_compression_enabled ? "Compressed" : "")
  available_regs = $panda_mask
  function("CreateStringFromCharArrayTlab#{suffix}".to_sym,
            params: {char_offset: 'u32', char_count: 'u32', char_array: 'ref', string_klass: 'ref'},
            regmap: $full_regmap,
            regalloc_set: available_regs,
            mode: [:FastPath]) {

    if Options.arch == :arm32
      Intrinsic(:UNREACHABLE).Terminator.void
      ReturnVoid().void
      next
    end

    # There is no check of the arguments against NullPointer as
    # it's done in the InstBuilder (see AddArgNullcheckIfNeeded)
    arr_offs := AddI(ShlI(Cast(char_offset).u64).Imm(1).u64).Imm(Constants::ARRAY_DATA_OFFSET).u64
    arr_data := Add(Cast(char_array).SrcType(Constants::COMPILER_REFERENCE).ptr, arr_offs).ptr

    # Allocate a new string
    if string_compression_enabled
      compressable := is_array_of_compressable_chars(arr_data, Cast(char_count).u64)
      If(compressable, 1).EQ.Likely.b {
        data_size1 := Cast(char_count).word
      } Else {
        data_size2 := Cast(ShlI(char_count).Imm(1).u32).word
      }
      data_size := Phi(data_size1, data_size2).word
    else
      data_size := Cast(ShlI(char_count).Imm(1).u32).word
    end
    new_str := allocate_string_tlab(string_klass, data_size)
    str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr

    # Copy data from char_array to the new string
    # String length field is set according to SetLength() from runtime/include/coretypes/string.h
    if string_compression_enabled
      If(compressable, 1).EQ.Likely.b {
        compress_u16_to_u8_chars(arr_data, str_data, Cast(char_count).u64)
        StoreI(new_str, ShlI(char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32
      } Else {
        copy_u16_chars(arr_data, str_data, Cast(char_count).u64)
        StoreI(new_str, OrI(ShlI(char_count).Imm(1).u32).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32
      }
    else
      copy_u16_chars(arr_data, str_data, Cast(char_count).u64)
      StoreI(new_str, char_count).Imm(Constants::STRING_LENGTH_OFFSET).u32
    end
    # String is supposed to be a constant object, so all its data should be visible by all threads
    Intrinsic(:DATA_MEMORY_BARRIER_FULL).void
    Return(new_str).ptr

  Label(:SlowPathEntrypoint)
    entrypoint = get_entrypoint_offset("CREATE_STRING_FROM_CHAR_ARRAY_SLOW_PATH")
    Intrinsic(:SLOW_PATH_ENTRY, char_offset, char_count, char_array).AddImm(entrypoint).MethodAsImm("CreateStringFromCharArray4ArgBridge").Terminator.ptr
    Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG
  }
end  # def GenerateCreateStringFromCharArrayTlab


def GenerateCreateStringFromZeroBasedCharArrayTlab(string_compression_enabled)
  suffix = (string_compression_enabled ? "Compressed" : "")
  available_regs = $panda_mask
  function("CreateStringFromZeroBasedCharArrayTlab#{suffix}".to_sym,
            params: {char_count: 'u32', char_array: 'ref', string_klass: 'ref'},
            regmap: $full_regmap,
            regalloc_set: available_regs,
            mode: [:FastPath]) {

    if Options.arch == :arm32
      Intrinsic(:UNREACHABLE).Terminator.void
      ReturnVoid().void
      next
    end

    # There is no check of the arguments against NullPointer as
    # it's done in the InstBuilder (see AddArgNullcheckIfNeeded)
    arr_data := Add(Cast(char_array).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::ARRAY_DATA_OFFSET).u64).ptr

    # Allocate a new string
    if string_compression_enabled
      compressable := is_array_of_compressable_chars(arr_data, Cast(char_count).u64)
      If(compressable, 1).EQ.Likely.b {
        data_size1 := Cast(char_count).word
      } Else {
        data_size2 := Cast(ShlI(char_count).Imm(1).u32).word
      }
      data_size := Phi(data_size1, data_size2).word
    else
      data_size := Cast(ShlI(char_count).Imm(1).u32).word
    end
    new_str := allocate_string_tlab(string_klass, data_size)
    str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr

    # Copy data from char_array to the new string
    # String length field is set according to SetLength() from runtime/include/coretypes/string.h
    if string_compression_enabled
      If(compressable, 1).EQ.Likely.b {
        compress_u16_to_u8_chars(arr_data, str_data, Cast(char_count).u64)
        StoreI(new_str, ShlI(char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32
      } Else {
        copy_u16_chars(arr_data, str_data, Cast(char_count).u64)
        StoreI(new_str, OrI(ShlI(char_count).Imm(1).u32).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32
      }
    else
      copy_u16_chars(arr_data, str_data, Cast(char_count).u64)
      StoreI(new_str, char_count).Imm(Constants::STRING_LENGTH_OFFSET).u32
    end
    # String is supposed to be a constant object, so all its data should be visible by all threads
    Intrinsic(:DATA_MEMORY_BARRIER_FULL).void
    Return(new_str).ptr

  Label(:SlowPathEntrypoint)
    entrypoint = get_entrypoint_offset("CREATE_STRING_FROM_ZERO_BASED_CHAR_ARRAY_SLOW_PATH")
    Intrinsic(:SLOW_PATH_ENTRY, char_count, char_array).AddImm(entrypoint).MethodAsImm("CreateStringFromZeroBasedCharArray3ArgBridge").Terminator.ptr
    Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG
  }
end  # def GenerateCreateStringFromZeroBasedCharArrayTlab


def GenerateSubstringFromStringTlab(string_compression_enabled)
  suffix = (string_compression_enabled ? "Compressed" : "")
  available_regs = $panda_mask
  function("SubStringFromStringTlab#{suffix}".to_sym,
           params: {str: 'ref', begin_index: 'i32', end_index: 'i32'},
           regmap: $full_regmap,
           regalloc_set: available_regs,
           mode: [:FastPath]) {

    if Options.arch == :arm32
      Intrinsic(:UNREACHABLE).Terminator.void
      ReturnVoid().void
      next
    end

    # Note, 'str' is checked against nullptr in the InstBuilder (see AddArgNullcheckIfNeeded)
    length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32
    if string_compression_enabled
      length := ShrI(length_packed).Imm(1).u32
    else
      length := length_packed
    end

    # If begin_index < 0, then it is assumed to be equal to zero
    If(begin_index, Cast(0).i32).LT.Unlikely.b {
      bx1 := Cast(0).i32
    }
    bx2 := Phi(begin_index, bx1).u32
    # If end_index < 0, then it is assumed to be equal to zero
    If(end_index, Cast(0).i32).LT.Unlikely.b {
      ex1 := Cast(0).i32
    }
    ex2 := Phi(end_index, ex1).u32
    # If begin_index > str.length(), then make it equal to str.length()
    If(bx2, length).A.Unlikely.b {
      bx3 := length
    }
    bx4 := Phi(bx2, bx3).u32
    # If end_index > str.length(), then make it equal to str.length()
    If(ex2, length).A.Unlikely.b {
      ex3 := length
    }
    ex4 := Phi(ex2, ex3).u32
    # If begin_index > end_index, then swap them.
    If(bx4, ex4).GT.Unlikely.b {
      bx5 := ex4
      ex5 := bx4
    }
    bx6 := Phi(bx4, bx5).u32
    ex6 := Phi(ex4, ex5).u32

    If(bx6, 0).EQ.Likely.b {
      If(ex6, length).EQ.Unlikely.b {
        Return(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr).ptr
      }
    }

    if string_compression_enabled
      not_compressed := AndI(length_packed).Imm(1).u32
      offset := Shl(bx6, not_compressed).u32
    else
      offset := ShlI(bx6).Imm(1).u32
    end

    src_str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).u64).ptr
    src_str_data := Add(src_str_data, Cast(offset).u64).ptr

    klass := load_class(str)
    char_count := Sub(ex6, bx6).u32

    # Allocate a new string
    if string_compression_enabled
      If(not_compressed, 1).EQ.Unlikely.b {
        compressable := is_array_of_compressable_chars(src_str_data, Cast(char_count).u64)
        If(compressable, 1).EQ.Likely.b {
          data_size1 := Cast(char_count).word
        } Else {
          data_size2 := Cast(ShlI(char_count).Imm(1).u32).word
        }
        data_size := Phi(data_size1, data_size2).word
        new_str := allocate_string_tlab(klass, data_size)
        new_str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr
        If(compressable, 1).EQ.Likely.b {
          compress_u16_to_u8_chars(src_str_data, new_str_data, Cast(char_count).u64)
          StoreI(new_str, ShlI(char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32
        } Else {
          copy_u16_chars(src_str_data, new_str_data, Cast(char_count).u64)
          StoreI(new_str, OrI(ShlI(char_count).Imm(1).u32).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32
        }
        # String is supposed to be a constant object, so all its data should be visible by all threads
        Intrinsic(:DATA_MEMORY_BARRIER_FULL).void
        Return(new_str).ptr
      } Else {
        # Source string is already compressed
        new_str := allocate_string_tlab(klass, Cast(char_count).word)
        new_str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr
        copy_u8_chars(src_str_data, new_str_data, Cast(char_count).u64)
        StoreI(new_str, ShlI(char_count).Imm(1).u32).Imm(Constants::STRING_LENGTH_OFFSET).u32
        # String is supposed to be a constant object, so all its data should be visible by all threads
        Intrinsic(:DATA_MEMORY_BARRIER_FULL).void
        Return(new_str).ptr
      }
    else
      data_size := Cast(ShlI(char_count).Imm(1).u32).word
      new_str := allocate_string_tlab(klass, data_size)
      new_str_data := Add(new_str, Cast(Constants::STRING_DATA_OFFSET).u64).ptr
      copy_u16_chars(src_str_data, new_str_data, Cast(char_count).u64)
      StoreI(new_str, char_count).Imm(Constants::STRING_LENGTH_OFFSET).u32
      # String is supposed to be a constant object, so all its data should be visible by all threads
      Intrinsic(:DATA_MEMORY_BARRIER_FULL).void
      Return(new_str).ptr
    end

  Label(:SlowPathEntrypoint)
    entrypoint = get_entrypoint_offset("SUB_STRING_FROM_STRING_SLOW_PATH")
    Intrinsic(:SLOW_PATH_ENTRY, str, begin_index, end_index).AddImm(entrypoint).MethodAsImm("SubStringFromStringOddSavedBridge").Terminator.ptr
    Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG
  }
end  # def GenerateSubstringFromStringTlab


def GenerateStringGetCharsTlab(string_compression_enabled)
  suffix = (string_compression_enabled ? "Compressed" : "")
  available_regs = $panda_mask
  function("StringGetCharsTlab#{suffix}".to_sym,
           params: {str: 'ref', begin_index: 'i32', end_index: 'i32', array_klass: 'ref'},
           regmap: $full_regmap,
           regalloc_set: available_regs,
           mode: [:FastPath]) {

    if Options.arch == :arm32
      Intrinsic(:UNREACHABLE).Terminator.void
      ReturnVoid().void
      next
    end

    If(begin_index, end_index).GT.Unlikely.b {
      Goto(:SlowPathEntrypoint)  # Out of range
    }
    If(begin_index, Cast(0).i32).LT.Unlikely.b {
      Goto(:SlowPathEntrypoint)  # Out of range
    }

    # Note, 'str' is checked against nullptr in the InstBuilder (see AddArgNullcheckIfNeeded)
    length := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32;
    if string_compression_enabled
      If(Cast(end_index).u32, ShrI(length).Imm(1).u32).A.Unlikely.b {
        Goto(:SlowPathEntrypoint)  # Out of range
      }
      not_compressed := AndI(length).Imm(1).u32
      offset := Shl(begin_index, not_compressed).u32
    else
      If(Cast(end_index).u32, length).A.Unlikely.b {
        Goto(:SlowPathEntrypoint)  # Out of range
      }
      not_compressed := Cast(1).u32
      offset := ShlI(begin_index).Imm(1).u32
    end

    src_str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).u64).ptr
    src_str_data := Add(src_str_data, Cast(offset).u64).ptr

    # Allocate a new array of u16 chars
    char_count := Sub(Cast(end_index).u32, Cast(begin_index).u32).u64
    new_arr := allocate_array_of_chars_tlab(array_klass, Cast(char_count).word)
    new_arr_data := Add(new_arr, Cast(Constants::ARRAY_DATA_OFFSET).u64).ptr
    If(not_compressed, Cast(0).u32).EQ.Likely.b {
      expand_u8_to_u16_chars(src_str_data, new_arr_data, char_count)
    }
    If(not_compressed, Cast(1).u32).EQ.Unlikely.b {
      copy_u16_chars(src_str_data, new_arr_data, char_count)
    }
    # String is supposed to be a constant object, so all its data should be visible by all threads
    Intrinsic(:DATA_MEMORY_BARRIER_FULL).void
    Return(new_arr).ptr

  Label(:SlowPathEntrypoint)
    entrypoint = get_entrypoint_offset("STRING_GET_CHARS_SLOW_PATH")
    Intrinsic(:SLOW_PATH_ENTRY, str, begin_index, end_index).AddImm(entrypoint).MethodAsImm("StringGetChars4ArgBridge").Terminator.ptr
    Intrinsic(:UNREACHABLE).Terminator.void if defines.DEBUG
  }
end  # def GenerateStringGetCharsTlab


###
# Compute string hashcode
#
scoped_macro(:u8_string_hashcode) do |str_data, char_count|
  hash := Cast(0).u32
  If(char_count, 0).A.Likely.b {
    i1 := Cast(0).u32
    imm31 := Cast(31).u32
Label(:Loop_hash)
    hash1 := Phi(hash, hash2).u32
    i := Phi(i1, i2).u32
    ch := Cast(Load(str_data, i).u8).SrcType(Constants::COMPILER_UINT8).u32
    hash2 := Add(Mul(hash1, imm31).u32, ch).u32
    i2 := AddI(i).Imm(1).u32
    If(i2, char_count).B.Likely.b {
      Goto(:Loop_hash)
    }
  }
  hashcode := Phi(hash, hash2).u32
end  # u8_string_hashcode

scoped_macro(:u16_string_hashcode) do |str_data, char_count|
  hash := Cast(0).u32
  If(char_count, 0).A.Likely.b {
    i1 := Cast(0).u64
    imm31 := Cast(31).u32
    stop := ShlI(Cast(char_count).u64).Imm(1).u64
Label(:Loop_hash)
    hash1 := Phi(hash, hash2).u32
    i := Phi(i1, i2).u64
    ch := Cast(Load(str_data, i).u16).SrcType(Constants::COMPILER_UINT16).u32
    hash2 := Add(Mul(hash1, imm31).u32, ch).u32
    i2 := AddI(i).Imm(2).u64
    If(i2, stop).B.Likely.b {
      Goto(:Loop_hash)
    }
  }
  hashcode := Phi(hash, hash2).u32
end  # u16_string_hashcode


def GenerateStringHashCode(string_compression_enabled)
  suffix = (string_compression_enabled ? "Compressed" : "")
if Options.arch == :arm64
  available_regs = $temps_mask + :arg0 + :callee0 + :callee1 + :callee2 + :callee3
else
  available_regs = $temps_mask + :arg0 + :callee0 + :caller0 + :caller1
end
  function("StringHashCode#{suffix}".to_sym,
            params: {str: 'ref'},
            regmap: $full_regmap,
            regalloc_set: available_regs,
            mode: [:FastPath]) {

    if Options.arch == :arm32
      Intrinsic(:UNREACHABLE).Terminator.void
      ReturnVoid().void
      next
    end

    # 1. There is no check of the argument against NullPointer as
    #    it's done in the InstBuilder (see AddArgNullcheckIfNeeded)
    # 2. Don't check if hashcode is equal to 0 as it's done in the codegen.

    str_data := Add(Cast(str).SrcType(Constants::COMPILER_REFERENCE).ptr, Cast(Constants::STRING_DATA_OFFSET).word).ptr
    length_packed := LoadI(str).Imm(Constants::STRING_LENGTH_OFFSET).u32
    if string_compression_enabled
      length := ShrI(length_packed).Imm(1).u32
      not_compressed := AndI(length_packed).Imm(1).u32
      If(not_compressed, 0).EQ.Likely.b {
        # String contains 8-bit chars
        h := u8_string_hashcode(str_data, length)
        StoreI(str, h).Imm(Constants::STRING_HASHCODE_OFFSET).u32
        Return(h).u32
      } Else {
        # String contains 16-bit chars
        h := u16_string_hashcode(str_data, length)
        StoreI(str, h).Imm(Constants::STRING_HASHCODE_OFFSET).u32
        Return(h).u32
      }
    else
      # String contains 16-bit chars
      h := u16_string_hashcode(str_data, length_packed)
      StoreI(str, h).Imm(Constants::STRING_HASHCODE_OFFSET).u32
      Return(h).u32
    end
  }
end  # def GenerateStringHashCode
